#include <iostream>
#include <fstream>
#include <string>
#include <string.h>
#include <stdlib.h>
//#include <sparsehash/unordered_map>
#include <unordered_map>
#include <time.h>
#include <vector>

#define FREQ_MIN_UNI 3
#define FREQ_MIN_BI 1

//using google::unordered_map;
using namespace std;
//using tr1::hash;
// or __gnu_cxx::hash, or maybe tr1::hash, depending on your OS

void loadUnigrams( ifstream &uniFile , unordered_map < string* , pair < unordered_map < string* , pair < vector < pair< string* , unsigned short int> > , unsigned short int > > , unsigned short int > > &mdic , unordered_map<int , unsigned short int> &freqIndex , unordered_map <string , string> &dict , vector<int> &frequencies){
	string line , s1;
	char* uni1 , * tok , * str;
	string* p1;
	int pos = 0 , freq = 0 , j = 0 , i = 0;

	while( getline(uniFile , line) ) {
		str = new char[line.length() + 1];
		strcpy( str , line.c_str() );
		uni1 = strtok(str , "\t");
		s1 = uni1;
		tok = strtok(NULL , "\n");
		freq = atoi(tok);
		//Solo guardamos los unigramas que aparecieron MAS de FREQ_MIN_UNI veces.
		if(freq > FREQ_MIN_UNI) {
			if( freqIndex.count(freq) != 1) {
				frequencies.push_back(freq);
				freqIndex[freq] = frequencies.size() - 1;
			}
			pos = freqIndex[freq];
			dict[s1] = s1;
			p1 = &dict[s1];
			mdic[p1].second = pos;
			j++;
		}
		delete [] str;
		i++;
		if(i % 100000 == 0) cout << "." << flush;
	}
	cout << endl << "Unigramas guardados: " << j << endl;
	cout << "Unigramas totales: " << i << endl;
}

void loadBigrams(ifstream &biFile , unordered_map < string* , pair < unordered_map < string* , pair < vector < pair< string* , unsigned short int> > , unsigned short int > > , unsigned short int > > &mdic , unordered_map<int , unsigned short int> &freqIndex , unordered_map <string , string> &dict , vector<int> &frequencies){
	string line;
	char* uni1 , * uni2 , * tok , * str;
	string* p1 , *p2;
	string s1 , s2 , s3;
	int i = 0 , j = 0 , freq = 0 , pos = 0;

	while( getline(biFile , line) ) {
		str = new char[line.length() + 1];
		strcpy( str , line.c_str() );
		uni1 = strtok(str , "_");
		s1 = uni1;
		uni2 = strtok(NULL , "\t");
		s2 = uni2;
		tok = strtok(NULL , "\n");
		freq = atoi(tok);
		//Solo guardamos los bigramas que aparecieron MAS de FREQ_MIN_BI veces y cuyos 2 unigramas ya estan cargados en los maps.
		if(freq > FREQ_MIN_BI && dict.count(s1) == 1 && dict.count(s2) == 1) {
			p1 = &dict[s1];
			p2 = &dict[s2];
			if(freqIndex.count(freq) != 1) {
				frequencies.push_back(freq);
				freqIndex[freq] = frequencies.size() - 1;
			}
			pos = freqIndex[freq];
			mdic[p1].first[p2].second = pos;
			j++;
		}
		delete [] str;
		i++;
		if(i % 100000 == 0) cout << "." << flush;
	}
	cout << endl << "Bigramas guardados: " << j << endl;
	cout << "Bigramas totales: " << i << endl;
}

void loadTrigrams(ifstream &triFile , unordered_map < string* , pair < unordered_map < string* , pair < vector < pair< string* , unsigned short int> > , unsigned short int > > , unsigned short int > > &mdic , unordered_map<int , unsigned short int> &freqIndex , unordered_map <string , string> &dict , vector<int> &frequencies){
	string line;
	char* uni1 , * uni2 , * uni3 , * tok , * str;
	string* p1 , *p2 , *p3;
	string s1 , s2 , s3;

	int i = 0 , j = 0 , freq = 0 , pos = 0;
	while( getline(triFile , line) ) {

		str = new char[line.length() + 1];
		strcpy( str , line.c_str() );
		uni1 = strtok(str , "_");
		s1 = uni1;
		uni2 = strtok(NULL , "_");
		s2 = uni2;
		uni3 = strtok(NULL , "\t");
		s3 = uni3;
		tok = strtok(NULL , "\n");
		freq = atoi(tok);

		//Solo guardamos los trigramas cuyos 3 unigramas ya estan guardados en los maps.
		if(dict.count(s1) == 1 && dict.count(s2) == 1 && dict.count(s3) == 1) {
			p1 = &dict[s1];
			p2 = &dict[s2];
			p3 = &dict[s3];
			//Solo guardamos los trigramas cuyos 2 bigramas ya estan guardados.
			if(mdic[p1].first.count(p2) == 1 && mdic[p2].first.count(p3) == 1) {
				if(freqIndex.count(freq) != 1 && frequencies[0] != freq) {
					frequencies.push_back(freq);
					freqIndex[freq] = frequencies.size() - 1;
				}
				pos = freqIndex[freq];
				mdic[p1].first[p2].first.push_back( pair<string* , short int>(p3 , pos) );
				j++;
			}

		}
		delete [] str;
		i++;
		if(i % 100000 == 0) cout << "." << flush;
	}
	cout << endl << "Trigramas guardados: " << j << endl;
	cout << "Trigramas totales: " << i << endl;
}

int main(int argc , char* argv[]) {
	double begin , end;
	begin = time(NULL);
	ifstream uniFile , biFile , triFile;
	uniFile.open ("unigramas");
	biFile.open("bigramas");
	triFile.open("trigramas");

	//Vector que guarda las distintas frequencies de cada ngrama.
	vector<int> frequencies;

	//Este map, dada una frecuencia nos da su posicion en el vector de frecuencias.
	unordered_map<int , unsigned short int> freqIndex;

	//En este map tenemos los unigramas, lo usamos para obtener los punteros a los strings.
	unordered_map <string , string> dict;

	//En este tenemos los unigamas/bigramas/trigramas con sus respectivas frecuencias.
	unordered_map < string* , pair < unordered_map < string* , pair < vector < pair< string* , unsigned short int> > , unsigned short int > > , unsigned short int > > mdic;

	cout << "UNIGRAMS" << endl;
	loadUnigrams(uniFile , mdic , freqIndex , dict , frequencies);
	cout << "BIGRAMS" << endl;
	loadBigrams(biFile , mdic , freqIndex , dict , frequencies);
	cout << "TRIGRAMS" << endl;
	loadTrigrams(triFile , mdic , freqIndex , dict , frequencies);

	end = time(NULL);
	cout << "Time elapsed: " << (int)difftime(end , begin) / 60 << "m " << (int)difftime(end , begin) % 60 << "s";
	cin.get();
}
